#!/usr/bin/env python
#coding=utf-8
import sys
from itertools import islice, cycle

def merge(bdfile, hdfile, wkfile, outfile):
    alldic = {}
    
    insertDic(ext(bdfile, 5, 2, 'gbk'), alldic)
    insertDic(ext(hdfile, 10, 5, 'utf8'), alldic)
    insertDic(ext(wkfile, 5, 2, 'utf8'), alldic)
    
    out = open(outfile,'w')
    
    existed = set()
    for item in sorted(alldic.items()):
        if item[0] in existed:
            continue
        existed.add(item[0])
        more = recursion(item[0], item[1], existed, alldic)
        more = getmain(more)
        if len(more) > 1:
            out.write('\t'.join(getmain(more)) + '\n')
    out.close()
        
def getmain(wordlist):
    dic = {}
    for x  in wordlist:
        dic[x] = dic.get(x, 0) + 1
    wordlist = [y for y in sorted([(dic[x], x) for x in dic], reverse=True) if len(y[1])<60]
    print '\t'.join([str(y[0]) for y in wordlist])
    return [y[1] for y in wordlist]    
        
def recursion(start, stlist, existed, alldic):
    more = stlist
    for x in stlist:
        if x in alldic:
            if x not in existed:
                existed.add(x)
                more.extend(recursion(x, alldic[x], existed, alldic))        
    return more
        
def insertDic(tyclist, alldic):
    for item in tyclist:
        if len(item) > 1:
            alldic.setdefault(item[0], []).extend(item[1:])
            for x in item[1:]:
                alldic.setdefault(x, []).append(item[0])
        
    
def ext(bdfile, glnum, tylinenum, enc):
    bdtyc = []
    infile = open(bdfile, 'r')
    myiter = iter(infile)
    group = list(islice(myiter, glnum))
    if enc == 'gbk':    
        try:
            group = [x.decode('gbk').encode('utf8') for x  in group]
        except:
            pass
    while len(group) == glnum:
        tyline = group[tylinenum-1].strip()[3:]
        if tyline:
            tmp = []
            tmp.append(group[0].strip()[3:])
            tmp.extend(tyline.split('\t'))
            bdtyc.append(tmp)
        group = list(islice(myiter, glnum))
        if enc == 'gbk':    
            try:
                group = [x.decode('gbk').encode('utf8') for x  in group]
            except:
                pass
        
    infile.close()
    return bdtyc

if __name__ == '__main__':        
    if len(sys.argv) != 5:
        print "usage: python xx.py bd.txt hd.txt wk.txt tyc_merge.txt"
    merge(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
    print 'ok'  
